In [26]:
sns.countplot(df['class'])
Out[26]:
<AxesSubplot:xlabel='class', ylabel='count'>
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import os
import warnings
warnings.filterwarnings("ignore")
%matplotlib inline
train = pd.read_csv("./Documents/PROJECTS/APS Failure detection/aps_failure_training_set.csv")
test = pd.read_csv("./Documents/PROJECTS/APS Failure detection/aps_failure_test_set.csv")
print(train.shape)
print(test.shape)
(60000, 171) (16000, 171)
train.head()
| class | aa_000 | ab_000 | ac_000 | ad_000 | ae_000 | af_000 | ag_000 | ag_001 | ag_002 | ... | ee_002 | ee_003 | ee_004 | ee_005 | ee_006 | ee_007 | ee_008 | ee_009 | ef_000 | eg_000 | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | neg | 76698 | na | 2130706438 | 280 | 0 | 0 | 0 | 0 | 0 | ... | 1240520 | 493384 | 721044 | 469792 | 339156 | 157956 | 73224 | 0 | 0 | 0 |
| 1 | neg | 33058 | na | 0 | na | 0 | 0 | 0 | 0 | 0 | ... | 421400 | 178064 | 293306 | 245416 | 133654 | 81140 | 97576 | 1500 | 0 | 0 |
| 2 | neg | 41040 | na | 228 | 100 | 0 | 0 | 0 | 0 | 0 | ... | 277378 | 159812 | 423992 | 409564 | 320746 | 158022 | 95128 | 514 | 0 | 0 |
| 3 | neg | 12 | 0 | 70 | 66 | 0 | 10 | 0 | 0 | 0 | ... | 240 | 46 | 58 | 44 | 10 | 0 | 0 | 0 | 4 | 32 |
| 4 | neg | 60874 | na | 1368 | 458 | 0 | 0 | 0 | 0 | 0 | ... | 622012 | 229790 | 405298 | 347188 | 286954 | 311560 | 433954 | 1218 | 0 | 0 |
5 rows × 171 columns
test.head()
| class | aa_000 | ab_000 | ac_000 | ad_000 | ae_000 | af_000 | ag_000 | ag_001 | ag_002 | ... | ee_002 | ee_003 | ee_004 | ee_005 | ee_006 | ee_007 | ee_008 | ee_009 | ef_000 | eg_000 | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | neg | 60 | 0 | 20 | 12 | 0 | 0 | 0 | 0 | 0 | ... | 1098 | 138 | 412 | 654 | 78 | 88 | 0 | 0 | 0 | 0 |
| 1 | neg | 82 | 0 | 68 | 40 | 0 | 0 | 0 | 0 | 0 | ... | 1068 | 276 | 1620 | 116 | 86 | 462 | 0 | 0 | 0 | 0 |
| 2 | neg | 66002 | 2 | 212 | 112 | 0 | 0 | 0 | 0 | 0 | ... | 495076 | 380368 | 440134 | 269556 | 1315022 | 153680 | 516 | 0 | 0 | 0 |
| 3 | neg | 59816 | na | 1010 | 936 | 0 | 0 | 0 | 0 | 0 | ... | 540820 | 243270 | 483302 | 485332 | 431376 | 210074 | 281662 | 3232 | 0 | 0 |
| 4 | neg | 1814 | na | 156 | 140 | 0 | 0 | 0 | 0 | 0 | ... | 7646 | 4144 | 18466 | 49782 | 3176 | 482 | 76 | 0 | 0 | 0 |
5 rows × 171 columns
data = pd.concat([train,test]).reset_index(drop=True)
data.shape
(76000, 171)
data.head()
| class | aa_000 | ab_000 | ac_000 | ad_000 | ae_000 | af_000 | ag_000 | ag_001 | ag_002 | ... | ee_002 | ee_003 | ee_004 | ee_005 | ee_006 | ee_007 | ee_008 | ee_009 | ef_000 | eg_000 | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | neg | 76698 | na | 2130706438 | 280 | 0 | 0 | 0 | 0 | 0 | ... | 1240520 | 493384 | 721044 | 469792 | 339156 | 157956 | 73224 | 0 | 0 | 0 |
| 1 | neg | 33058 | na | 0 | na | 0 | 0 | 0 | 0 | 0 | ... | 421400 | 178064 | 293306 | 245416 | 133654 | 81140 | 97576 | 1500 | 0 | 0 |
| 2 | neg | 41040 | na | 228 | 100 | 0 | 0 | 0 | 0 | 0 | ... | 277378 | 159812 | 423992 | 409564 | 320746 | 158022 | 95128 | 514 | 0 | 0 |
| 3 | neg | 12 | 0 | 70 | 66 | 0 | 10 | 0 | 0 | 0 | ... | 240 | 46 | 58 | 44 | 10 | 0 | 0 | 0 | 4 | 32 |
| 4 | neg | 60874 | na | 1368 | 458 | 0 | 0 | 0 | 0 | 0 | ... | 622012 | 229790 | 405298 | 347188 | 286954 | 311560 | 433954 | 1218 | 0 | 0 |
5 rows × 171 columns
data['class'].unique()
array(['neg', 'pos'], dtype=object)
data['class'].value_counts()
neg 74625 pos 1375 Name: class, dtype: int64
data.info()
<class 'pandas.core.frame.DataFrame'> RangeIndex: 76000 entries, 0 to 75999 Columns: 171 entries, class to eg_000 dtypes: int64(1), object(170) memory usage: 99.2+ MB
data.describe()
| aa_000 | |
|---|---|
| count | 7.600000e+04 |
| mean | 6.115976e+04 |
| std | 2.647366e+05 |
| min | 0.000000e+00 |
| 25% | 8.600000e+02 |
| 50% | 3.081300e+04 |
| 75% | 4.884000e+04 |
| max | 4.294967e+07 |
data.describe(include='O')
| class | ab_000 | ac_000 | ad_000 | ae_000 | af_000 | ag_000 | ag_001 | ag_002 | ag_003 | ... | ee_002 | ee_003 | ee_004 | ee_005 | ee_006 | ee_007 | ee_008 | ee_009 | ef_000 | eg_000 | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| count | 76000 | 76000 | 76000 | 76000 | 76000 | 76000 | 76000 | 76000 | 76000 | 76000 | ... | 76000 | 76000 | 76000 | 76000 | 76000 | 76000 | 76000 | 76000 | 76000 | 76000 |
| unique | 2 | 31 | 2230 | 2028 | 356 | 459 | 199 | 796 | 2953 | 9384 | ... | 40697 | 37292 | 41658 | 43052 | 37561 | 35860 | 28413 | 10948 | 31 | 56 |
| top | neg | na | 0 | na | 0 | 0 | 0 | 0 | 0 | 0 | ... | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
| freq | 74625 | 58692 | 11041 | 18842 | 70307 | 70217 | 74882 | 74162 | 71135 | 59336 | ... | 1720 | 1963 | 2266 | 3549 | 5609 | 9959 | 21780 | 40295 | 72188 | 71901 |
4 rows × 170 columns
data.duplicated().sum()
0
df = data.copy()
df.shape
(76000, 171)
df.info(max_cols=172, show_counts=True)
<class 'pandas.core.frame.DataFrame'> RangeIndex: 76000 entries, 0 to 75999 Data columns (total 171 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- 0 class 76000 non-null object 1 aa_000 76000 non-null int64 2 ab_000 76000 non-null object 3 ac_000 76000 non-null object 4 ad_000 76000 non-null object 5 ae_000 76000 non-null object 6 af_000 76000 non-null object 7 ag_000 76000 non-null object 8 ag_001 76000 non-null object 9 ag_002 76000 non-null object 10 ag_003 76000 non-null object 11 ag_004 76000 non-null object 12 ag_005 76000 non-null object 13 ag_006 76000 non-null object 14 ag_007 76000 non-null object 15 ag_008 76000 non-null object 16 ag_009 76000 non-null object 17 ah_000 76000 non-null object 18 ai_000 76000 non-null object 19 aj_000 76000 non-null object 20 ak_000 76000 non-null object 21 al_000 76000 non-null object 22 am_0 76000 non-null object 23 an_000 76000 non-null object 24 ao_000 76000 non-null object 25 ap_000 76000 non-null object 26 aq_000 76000 non-null object 27 ar_000 76000 non-null object 28 as_000 76000 non-null object 29 at_000 76000 non-null object 30 au_000 76000 non-null object 31 av_000 76000 non-null object 32 ax_000 76000 non-null object 33 ay_000 76000 non-null object 34 ay_001 76000 non-null object 35 ay_002 76000 non-null object 36 ay_003 76000 non-null object 37 ay_004 76000 non-null object 38 ay_005 76000 non-null object 39 ay_006 76000 non-null object 40 ay_007 76000 non-null object 41 ay_008 76000 non-null object 42 ay_009 76000 non-null object 43 az_000 76000 non-null object 44 az_001 76000 non-null object 45 az_002 76000 non-null object 46 az_003 76000 non-null object 47 az_004 76000 non-null object 48 az_005 76000 non-null object 49 az_006 76000 non-null object 50 az_007 76000 non-null object 51 az_008 76000 non-null object 52 az_009 76000 non-null object 53 ba_000 76000 non-null object 54 ba_001 76000 non-null object 55 ba_002 76000 non-null object 56 ba_003 76000 non-null object 57 ba_004 76000 non-null object 58 ba_005 76000 non-null object 59 ba_006 76000 non-null object 60 ba_007 76000 non-null object 61 ba_008 76000 non-null object 62 ba_009 76000 non-null object 63 bb_000 76000 non-null object 64 bc_000 76000 non-null object 65 bd_000 76000 non-null object 66 be_000 76000 non-null object 67 bf_000 76000 non-null object 68 bg_000 76000 non-null object 69 bh_000 76000 non-null object 70 bi_000 76000 non-null object 71 bj_000 76000 non-null object 72 bk_000 76000 non-null object 73 bl_000 76000 non-null object 74 bm_000 76000 non-null object 75 bn_000 76000 non-null object 76 bo_000 76000 non-null object 77 bp_000 76000 non-null object 78 bq_000 76000 non-null object 79 br_000 76000 non-null object 80 bs_000 76000 non-null object 81 bt_000 76000 non-null object 82 bu_000 76000 non-null object 83 bv_000 76000 non-null object 84 bx_000 76000 non-null object 85 by_000 76000 non-null object 86 bz_000 76000 non-null object 87 ca_000 76000 non-null object 88 cb_000 76000 non-null object 89 cc_000 76000 non-null object 90 cd_000 76000 non-null object 91 ce_000 76000 non-null object 92 cf_000 76000 non-null object 93 cg_000 76000 non-null object 94 ch_000 76000 non-null object 95 ci_000 76000 non-null object 96 cj_000 76000 non-null object 97 ck_000 76000 non-null object 98 cl_000 76000 non-null object 99 cm_000 76000 non-null object 100 cn_000 76000 non-null object 101 cn_001 76000 non-null object 102 cn_002 76000 non-null object 103 cn_003 76000 non-null object 104 cn_004 76000 non-null object 105 cn_005 76000 non-null object 106 cn_006 76000 non-null object 107 cn_007 76000 non-null object 108 cn_008 76000 non-null object 109 cn_009 76000 non-null object 110 co_000 76000 non-null object 111 cp_000 76000 non-null object 112 cq_000 76000 non-null object 113 cr_000 76000 non-null object 114 cs_000 76000 non-null object 115 cs_001 76000 non-null object 116 cs_002 76000 non-null object 117 cs_003 76000 non-null object 118 cs_004 76000 non-null object 119 cs_005 76000 non-null object 120 cs_006 76000 non-null object 121 cs_007 76000 non-null object 122 cs_008 76000 non-null object 123 cs_009 76000 non-null object 124 ct_000 76000 non-null object 125 cu_000 76000 non-null object 126 cv_000 76000 non-null object 127 cx_000 76000 non-null object 128 cy_000 76000 non-null object 129 cz_000 76000 non-null object 130 da_000 76000 non-null object 131 db_000 76000 non-null object 132 dc_000 76000 non-null object 133 dd_000 76000 non-null object 134 de_000 76000 non-null object 135 df_000 76000 non-null object 136 dg_000 76000 non-null object 137 dh_000 76000 non-null object 138 di_000 76000 non-null object 139 dj_000 76000 non-null object 140 dk_000 76000 non-null object 141 dl_000 76000 non-null object 142 dm_000 76000 non-null object 143 dn_000 76000 non-null object 144 do_000 76000 non-null object 145 dp_000 76000 non-null object 146 dq_000 76000 non-null object 147 dr_000 76000 non-null object 148 ds_000 76000 non-null object 149 dt_000 76000 non-null object 150 du_000 76000 non-null object 151 dv_000 76000 non-null object 152 dx_000 76000 non-null object 153 dy_000 76000 non-null object 154 dz_000 76000 non-null object 155 ea_000 76000 non-null object 156 eb_000 76000 non-null object 157 ec_00 76000 non-null object 158 ed_000 76000 non-null object 159 ee_000 76000 non-null object 160 ee_001 76000 non-null object 161 ee_002 76000 non-null object 162 ee_003 76000 non-null object 163 ee_004 76000 non-null object 164 ee_005 76000 non-null object 165 ee_006 76000 non-null object 166 ee_007 76000 non-null object 167 ee_008 76000 non-null object 168 ee_009 76000 non-null object 169 ef_000 76000 non-null object 170 eg_000 76000 non-null object dtypes: int64(1), object(170) memory usage: 99.2+ MB
num_f = [feature for feature in df.columns if df[feature].dtype != 'O']
cat_f = [feature for feature in df.columns if df[feature].dtype == 'O']
print("we have {} numerical features: {}".format(len(num_f), num_f))
print("we have {} categorical features: {}".format(len(cat_f), cat_f))
we have 1 numerical features: ['aa_000'] we have 170 categorical features: ['class', 'ab_000', 'ac_000', 'ad_000', 'ae_000', 'af_000', 'ag_000', 'ag_001', 'ag_002', 'ag_003', 'ag_004', 'ag_005', 'ag_006', 'ag_007', 'ag_008', 'ag_009', 'ah_000', 'ai_000', 'aj_000', 'ak_000', 'al_000', 'am_0', 'an_000', 'ao_000', 'ap_000', 'aq_000', 'ar_000', 'as_000', 'at_000', 'au_000', 'av_000', 'ax_000', 'ay_000', 'ay_001', 'ay_002', 'ay_003', 'ay_004', 'ay_005', 'ay_006', 'ay_007', 'ay_008', 'ay_009', 'az_000', 'az_001', 'az_002', 'az_003', 'az_004', 'az_005', 'az_006', 'az_007', 'az_008', 'az_009', 'ba_000', 'ba_001', 'ba_002', 'ba_003', 'ba_004', 'ba_005', 'ba_006', 'ba_007', 'ba_008', 'ba_009', 'bb_000', 'bc_000', 'bd_000', 'be_000', 'bf_000', 'bg_000', 'bh_000', 'bi_000', 'bj_000', 'bk_000', 'bl_000', 'bm_000', 'bn_000', 'bo_000', 'bp_000', 'bq_000', 'br_000', 'bs_000', 'bt_000', 'bu_000', 'bv_000', 'bx_000', 'by_000', 'bz_000', 'ca_000', 'cb_000', 'cc_000', 'cd_000', 'ce_000', 'cf_000', 'cg_000', 'ch_000', 'ci_000', 'cj_000', 'ck_000', 'cl_000', 'cm_000', 'cn_000', 'cn_001', 'cn_002', 'cn_003', 'cn_004', 'cn_005', 'cn_006', 'cn_007', 'cn_008', 'cn_009', 'co_000', 'cp_000', 'cq_000', 'cr_000', 'cs_000', 'cs_001', 'cs_002', 'cs_003', 'cs_004', 'cs_005', 'cs_006', 'cs_007', 'cs_008', 'cs_009', 'ct_000', 'cu_000', 'cv_000', 'cx_000', 'cy_000', 'cz_000', 'da_000', 'db_000', 'dc_000', 'dd_000', 'de_000', 'df_000', 'dg_000', 'dh_000', 'di_000', 'dj_000', 'dk_000', 'dl_000', 'dm_000', 'dn_000', 'do_000', 'dp_000', 'dq_000', 'dr_000', 'ds_000', 'dt_000', 'du_000', 'dv_000', 'dx_000', 'dy_000', 'dz_000', 'ea_000', 'eb_000', 'ec_00', 'ed_000', 'ee_000', 'ee_001', 'ee_002', 'ee_003', 'ee_004', 'ee_005', 'ee_006', 'ee_007', 'ee_008', 'ee_009', 'ef_000', 'eg_000']
for col in cat_f:
print(df[col].value_counts(normalize=True)*100)
print('---------------------------')
neg 98.190789
pos 1.809211
Name: class, dtype: float64
---------------------------
na 77.226316
0 18.213158
2 3.268421
4 0.706579
6 0.239474
8 0.119737
10 0.067105
12 0.047368
14 0.025000
16 0.011842
24 0.011842
18 0.007895
20 0.006579
26 0.006579
22 0.005263
38 0.005263
34 0.003947
100 0.003947
58 0.003947
28 0.003947
32 0.002632
52 0.001316
90 0.001316
134 0.001316
204 0.001316
36 0.001316
48 0.001316
44 0.001316
46 0.001316
68 0.001316
30 0.001316
Name: ab_000, dtype: float64
---------------------------
0 14.527632
2130706432 10.343421
na 5.606579
2130706434 2.036842
2 1.280263
...
3032 0.001316
3264 0.001316
3416 0.001316
7702 0.001316
3490 0.001316
Name: ac_000, Length: 2230, dtype: float64
---------------------------
na 24.792105
0 3.326316
2 1.560526
4 1.481579
6 1.443421
...
6486 0.001316
3938 0.001316
4518 0.001316
2258 0.001316
2226 0.001316
Name: ad_000, Length: 2028, dtype: float64
---------------------------
0 92.509211
na 4.197368
2 0.982895
4 0.173684
8 0.107895
...
366 0.001316
188 0.001316
416 0.001316
894 0.001316
2088 0.001316
Name: ae_000, Length: 356, dtype: float64
---------------------------
0 92.390789
na 4.197368
2 0.959211
4 0.189474
6 0.135526
...
954 0.001316
208 0.001316
336 0.001316
13562 0.001316
422 0.001316
Name: af_000, Length: 459, dtype: float64
---------------------------
0 98.528947
na 1.131579
2 0.014474
11912 0.006579
4 0.006579
...
676 0.001316
34 0.001316
10682 0.001316
26884 0.001316
212 0.001316
Name: ag_000, Length: 199, dtype: float64
---------------------------
0 97.581579
na 1.131579
2 0.017105
12 0.007895
530916 0.006579
...
5456 0.001316
554 0.001316
153344 0.001316
1150462 0.001316
4362 0.001316
Name: ag_001, Length: 796, dtype: float64
---------------------------
0 93.598684
na 1.131579
342 0.013158
16 0.013158
12 0.013158
...
284 0.001316
22990 0.001316
9324 0.001316
8670 0.001316
13320 0.001316
Name: ag_002, Length: 2953, dtype: float64
---------------------------
0 78.073684
na 1.131579
22 0.036842
30 0.036842
2 0.035526
...
1326224 0.001316
9369826 0.001316
5399416 0.001316
432658 0.001316
2388752 0.001316
Name: ag_003, Length: 9384, dtype: float64
---------------------------
0 2.164474
na 1.131579
6 0.509211
2 0.442105
4 0.442105
...
32614 0.001316
9404 0.001316
51104 0.001316
179574 0.001316
5486 0.001316
Name: ag_004, Length: 26855, dtype: float64
---------------------------
0 3.210526
na 1.131579
554 0.022368
2016 0.017105
1700 0.017105
...
432162 0.001316
29090 0.001316
19873666 0.001316
606156 0.001316
704012 0.001316
Name: ag_005, Length: 48328, dtype: float64
---------------------------
0 2.057895
na 1.131579
1002 0.032895
1046 0.028947
1068 0.027632
...
1723974 0.001316
374960 0.001316
1631462 0.001316
1378442 0.001316
3220278 0.001316
Name: ag_006, Length: 48481, dtype: float64
---------------------------
0 26.334211
na 1.131579
4 0.034211
18 0.032895
12 0.031579
...
505052 0.001316
1468188 0.001316
34198 0.001316
620904 0.001316
1297498 0.001316
Name: ag_007, Length: 38290, dtype: float64
---------------------------
0 41.809211
na 1.131579
4 0.088158
82 0.039474
12 0.035526
...
79378 0.001316
15816 0.001316
120588 0.001316
38570 0.001316
76170 0.001316
Name: ag_008, Length: 21564, dtype: float64
---------------------------
0 67.756579
na 1.131579
4 0.221053
2 0.123684
6 0.100000
...
9854 0.001316
30630 0.001316
15074 0.001316
4582 0.001316
7206 0.001316
Name: ag_009, Length: 5924, dtype: float64
---------------------------
na 1.078947
0 0.214474
1868 0.023684
1436 0.021053
2228 0.019737
...
1675550 0.001316
1242876 0.001316
11382 0.001316
7342 0.001316
2276100 0.001316
Name: ah_000, Length: 52009, dtype: float64
---------------------------
0 89.194737
na 1.042105
4466 0.009211
1166 0.009211
1444 0.009211
...
15552 0.001316
127224 0.001316
35528 0.001316
17966 0.001316
170808 0.001316
Name: ai_000, Length: 4805, dtype: float64
---------------------------
0 78.151316
na 1.042105
58 0.530263
54 0.492105
56 0.475000
...
2164 0.001316
924 0.001316
662 0.001316
1284 0.001316
23862 0.001316
Name: aj_000, Length: 1032, dtype: float64
---------------------------
0 92.006579
na 7.365789
598 0.215789
596 0.043421
1196 0.021053
...
11796 0.001316
1630588 0.001316
378 0.001316
1840 0.001316
91940 0.001316
Name: ak_000, Length: 195, dtype: float64
---------------------------
0 62.352632
na 1.067105
54 0.080263
158 0.061842
178 0.060526
...
8070 0.001316
220072 0.001316
590992 0.001316
1009872 0.001316
1654760 0.001316
Name: al_000, Length: 10457, dtype: float64
---------------------------
0 62.042105
na 1.042105
1196 0.110526
1192 0.078947
1184 0.078947
...
50160 0.001316
196186 0.001316
13128 0.001316
12472 0.001316
2779232 0.001316
Name: am_0, Length: 12215, dtype: float64
---------------------------
na 1.067105
0 0.211842
11742 0.013158
99428 0.011842
10542 0.010526
...
81480 0.001316
2380170 0.001316
49366 0.001316
6639462 0.001316
26026 0.001316
Name: an_000, Length: 57988, dtype: float64
---------------------------
na 0.988158
0 0.227632
87122 0.011842
5896 0.011842
11286 0.010526
...
30468 0.001316
11125654 0.001316
85714 0.001316
2032772 0.001316
13504 0.001316
Name: ao_000, Length: 57387, dtype: float64
---------------------------
na 1.067105
0 0.185526
8080 0.013158
31576 0.013158
13100 0.011842
...
1313492 0.001316
1253482 0.001316
1477154 0.001316
378508 0.001316
61018 0.001316
Name: ap_000, Length: 53132, dtype: float64
---------------------------
na 0.988158
0 0.361842
336 0.043421
398 0.043421
404 0.042105
...
267290 0.001316
13280 0.001316
261500 0.001316
260338 0.001316
669608 0.001316
Name: aq_000, Length: 41141, dtype: float64
---------------------------
0 91.453947
na 4.588158
2 1.365789
4 0.692105
6 0.461842
...
222 0.001316
178 0.001316
98 0.001316
184 0.001316
188 0.001316
Name: ar_000, Length: 73, dtype: float64
---------------------------
0 98.918421
na 1.042105
13752 0.002632
1246190 0.001316
3550 0.001316
3662 0.001316
1159402 0.001316
3314194 0.001316
13792 0.001316
42940 0.001316
6383704 0.001316
10696 0.001316
647060 0.001316
1150508 0.001316
489122 0.001316
790 0.001316
4 0.001316
693990 0.001316
368184 0.001316
201306 0.001316
63712 0.001316
357260 0.001316
128844 0.001316
320346 0.001316
113724 0.001316
6268 0.001316
1156 0.001316
1655240 0.001316
31518 0.001316
21748 0.001316
145054 0.001316
Name: as_000, dtype: float64
---------------------------
0 88.898684
na 1.042105
756 0.022368
1014 0.019737
686 0.018421
...
7936 0.001316
19368 0.001316
4520 0.001316
678 0.001316
972 0.001316
Name: at_000, Length: 4026, dtype: float64
---------------------------
0 98.853947
na 1.042105
249166 0.003947
1324 0.003947
96986 0.002632
...
1094074 0.001316
26212 0.001316
588396 0.001316
10082 0.001316
358 0.001316
Name: au_000, Length: 72, dtype: float64
---------------------------
0 16.663158
na 4.194737
4 1.488158
6 1.419737
2 1.407895
...
5944 0.001316
8764 0.001316
11282 0.001316
15644 0.001316
6374 0.001316
Name: av_000, Length: 4221, dtype: float64
---------------------------
0 16.988158
na 4.196053
6 1.518421
4 1.497368
10 1.438158
...
2844 0.001316
8836 0.001316
12442 0.001316
2840 0.001316
2668 0.001316
Name: ax_000, Length: 2416, dtype: float64
---------------------------
0 97.905263
na 1.135526
2 0.019737
36276 0.005263
180266 0.005263
...
42004 0.001316
35306 0.001316
35814 0.001316
38966 0.001316
148832 0.001316
Name: ay_000, Length: 576, dtype: float64
---------------------------
0 96.965789
na 1.135526
664 0.006579
1358 0.005263
942 0.005263
...
14924 0.001316
13926 0.001316
4794 0.001316
2842 0.001316
1872 0.001316
Name: ay_001, Length: 1111, dtype: float64
---------------------------
0 96.922368
na 1.135526
1372508 0.005263
8852 0.005263
21788 0.005263
...
11252 0.001316
5016 0.001316
24486 0.001316
7506 0.001316
5270 0.001316
Name: ay_002, Length: 1195, dtype: float64
---------------------------
0 96.792105
na 1.135526
13998 0.006579
17278 0.006579
29174 0.005263
...
5550 0.001316
26930 0.001316
16192 0.001316
11126 0.001316
6774 0.001316
Name: ay_003, Length: 1250, dtype: float64
---------------------------
0 95.240789
na 1.135526
2 0.011842
196 0.009211
1352 0.009211
...
2062 0.001316
8066 0.001316
5790 0.001316
12378 0.001316
5062 0.001316
Name: ay_004, Length: 2127, dtype: float64
---------------------------
0 50.496053
na 1.135526
2 0.028947
54 0.017105
96 0.015789
...
174210 0.001316
586070 0.001316
120420 0.001316
113034 0.001316
214014 0.001316
Name: ay_005, Length: 22881, dtype: float64
---------------------------
0 27.661842
na 1.135526
52858 0.011842
2 0.010526
169962 0.010526
...
620450 0.001316
15010 0.001316
888996 0.001316
1292178 0.001316
3876498 0.001316
Name: ay_006, Length: 41539, dtype: float64
---------------------------
0 12.394737
na 1.135526
2 0.050000
4 0.022368
830 0.021053
...
162956 0.001316
1197464 0.001316
90486 0.001316
28298 0.001316
14678 0.001316
Name: ay_007, Length: 46023, dtype: float64
---------------------------
0 8.156579
na 1.135526
2 0.792105
4 0.193421
6 0.126316
...
318192 0.001316
397236 0.001316
167448 0.001316
147660 0.001316
351926 0.001316
Name: ay_008, Length: 44764, dtype: float64
---------------------------
0 97.936842
na 1.135526
2 0.038158
6 0.013158
4 0.013158
...
29404 0.001316
19920 0.001316
470660 0.001316
33884 0.001316
38752 0.001316
Name: ay_009, Length: 569, dtype: float64
---------------------------
0 1.828947
na 1.135526
54 0.261842
52 0.251316
70 0.247368
...
326292 0.001316
33410 0.001316
6516 0.001316
16052 0.001316
37452 0.001316
Name: az_000, Length: 10291, dtype: float64
---------------------------
0 2.464474
10 1.298684
12 1.281579
6 1.267105
14 1.264474
...
5220 0.001316
10936 0.001316
9388 0.001316
53260 0.001316
29656 0.001316
Name: az_001, Length: 8288, dtype: float64
---------------------------
0 2.147368
na 1.135526
26 0.898684
18 0.894737
14 0.872368
...
35698 0.001316
30626 0.001316
10848 0.001316
124600 0.001316
40232 0.001316
Name: az_002, Length: 10237, dtype: float64
---------------------------
0 1.975000
na 1.135526
24 0.585526
16 0.563158
26 0.539474
...
863914 0.001316
367228 0.001316
113848 0.001316
36016 0.001316
100040 0.001316
Name: az_003, Length: 24772, dtype: float64
---------------------------
0 1.915789
na 1.135526
12 0.255263
8 0.213158
20 0.213158
...
211042 0.001316
1966444 0.001316
2040166 0.001316
10198 0.001316
321282 0.001316
Name: az_004, Length: 40464, dtype: float64
---------------------------
0 1.884211
na 1.135526
140 0.023684
194 0.021053
250 0.021053
...
331922 0.001316
1868740 0.001316
41912 0.001316
4162390 0.001316
25512 0.001316
Name: az_005, Length: 53356, dtype: float64
---------------------------
0 14.350000
2 3.672368
4 2.482895
6 1.892105
8 1.535526
...
46704 0.001316
393688 0.001316
1080614 0.001316
21092 0.001316
147494 0.001316
Name: az_006, Length: 14148, dtype: float64
---------------------------
0 75.130263
2 5.946053
4 2.290789
6 1.246053
na 1.135526
...
1718 0.001316
3740 0.001316
7854 0.001316
114038 0.001316
1946 0.001316
Name: az_007, Length: 4564, dtype: float64
---------------------------
0 90.359211
2 2.360526
na 1.135526
4 0.869737
6 0.453947
...
177148 0.001316
3816 0.001316
8250 0.001316
984 0.001316
810 0.001316
Name: az_008, Length: 1454, dtype: float64
---------------------------
0 95.744737
na 1.135526
2 1.061842
4 0.365789
6 0.219737
...
1378 0.001316
324 0.001316
222 0.001316
2934 0.001316
206 0.001316
Name: az_009, Length: 387, dtype: float64
---------------------------
0 1.819737
na 1.159211
6280 0.014474
47074 0.013158
5148 0.013158
...
1922352 0.001316
1134122 0.001316
17722 0.001316
117266 0.001316
1591426 0.001316
Name: ba_000, Length: 53547, dtype: float64
---------------------------
0 1.903947
na 1.159211
230 0.050000
228 0.050000
248 0.050000
...
731536 0.001316
33874 0.001316
1042430 0.001316
791650 0.001316
2742 0.001316
Name: ba_001, Length: 47872, dtype: float64
---------------------------
0 1.957895
na 1.159211
78 0.177632
90 0.175000
112 0.172368
...
357820 0.001316
980542 0.001316
240544 0.001316
428230 0.001316
346644 0.001316
Name: ba_002, Length: 42463, dtype: float64
---------------------------
0 2.003947
na 1.159211
58 0.239474
62 0.219737
56 0.215789
...
4521400 0.001316
235386 0.001316
132558 0.001316
512952 0.001316
248082 0.001316
Name: ba_003, Length: 38701, dtype: float64
---------------------------
0 2.055263
na 1.159211
32 0.344737
44 0.342105
24 0.340789
...
345180 0.001316
124978 0.001316
6000820 0.001316
38156 0.001316
183738 0.001316
Name: ba_004, Length: 36077, dtype: float64
---------------------------
0 2.202632
na 1.159211
22 0.430263
26 0.386842
28 0.372368
...
1624322 0.001316
95870 0.001316
106894 0.001316
68816 0.001316
177698 0.001316
Name: ba_005, Length: 34938, dtype: float64
---------------------------
0 3.303947
na 1.159211
8 0.319737
2 0.303947
14 0.298684
...
117336 0.001316
41488 0.001316
464418 0.001316
316414 0.001316
299794 0.001316
Name: ba_006, Length: 34659, dtype: float64
---------------------------
0 13.317105
na 1.159211
2 0.960526
4 0.761842
6 0.647368
...
64692 0.001316
273228 0.001316
255840 0.001316
37726 0.001316
1141712 0.001316
Name: ba_007, Length: 29877, dtype: float64
---------------------------
0 41.323684
2 1.297368
na 1.159211
4 0.894737
8 0.744737
...
8382 0.001316
1007450 0.001316
18448 0.001316
353234 0.001316
1613458 0.001316
Name: ba_008, Length: 13926, dtype: float64
---------------------------
0 65.918421
na 1.159211
2 1.103947
4 0.740789
6 0.603947
...
74952 0.001316
955980 0.001316
129726 0.001316
9016 0.001316
1630316 0.001316
Name: ba_009, Length: 8089, dtype: float64
---------------------------
na 1.078947
0 0.175000
131080 0.011842
18258 0.010526
2537146 0.009211
...
48046 0.001316
48478984 0.001316
124412 0.001316
6762168 0.001316
87120 0.001316
Name: bb_000, Length: 59706, dtype: float64
---------------------------
0 32.006579
2 5.277632
na 4.590789
4 3.068421
6 2.010526
...
6960 0.001316
2280 0.001316
32166 0.001316
2672 0.001316
6476 0.001316
Name: bc_000, Length: 3123, dtype: float64
---------------------------
0 6.702632
4 5.134211
2 4.897368
na 4.593421
6 3.938158
...
26782 0.001316
20942 0.001316
8942 0.001316
4172 0.001316
8936 0.001316
Name: bd_000, Length: 3945, dtype: float64
---------------------------
0 11.469737
na 4.201316
2 3.875000
4 2.182895
6 1.506579
...
7400 0.001316
24818 0.001316
3878 0.001316
13792 0.001316
4120 0.001316
Name: be_000, Length: 4368, dtype: float64
---------------------------
0 42.096053
2 8.080263
4 4.880263
na 4.196053
6 3.768421
...
7124 0.001316
744 0.001316
986 0.001316
1002 0.001316
3286 0.001316
Name: bf_000, Length: 1220, dtype: float64
---------------------------
na 1.067105
0 0.214474
1868 0.023684
1436 0.021053
2232 0.019737
...
11382 0.001316
7342 0.001316
104252 0.001316
52692 0.001316
2276100 0.001316
Name: bg_000, Length: 52013, dtype: float64
---------------------------
na 1.067105
0 0.211842
134 0.202632
138 0.186842
140 0.185526
...
84968 0.001316
216070 0.001316
41020 0.001316
113180 0.001316
68662 0.001316
Name: bh_000, Length: 29069, dtype: float64
---------------------------
na 0.988158
0 0.190789
2 0.019737
8112 0.014474
5534 0.014474
...
535174 0.001316
359680 0.001316
241010 0.001316
181424 0.001316
46216 0.001316
Name: bi_000, Length: 49751, dtype: float64
---------------------------
na 0.988158
0 0.222368
1856 0.025000
2044 0.023684
2458 0.021053
...
47810 0.001316
670438 0.001316
291110 0.001316
1027228 0.001316
14332 0.001316
Name: bj_000, Length: 45122, dtype: float64
---------------------------
na 38.326316
1310700 3.215789
0 0.807895
161880 0.025000
212520 0.023684
...
407520 0.001316
368180 0.001316
610480 0.001316
334020 0.001316
403880 0.001316
Name: bk_000, Length: 14030, dtype: float64
---------------------------
na 45.398684
1310700 4.646053
0 1.676316
156260 0.022368
174560 0.019737
...
504040 0.001316
421660 0.001316
329680 0.001316
316880 0.001316
128480 0.001316
Name: bl_000, Length: 13086, dtype: float64
---------------------------
na 65.914474
1310700 5.294737
0 2.022368
193240 0.019737
198100 0.014474
...
325500 0.001316
216840 0.001316
386380 0.001316
717220 0.001316
314260 0.001316
Name: bm_000, Length: 10255, dtype: float64
---------------------------
na 73.318421
1310700 5.811842
0 2.363158
182360 0.014474
254940 0.013158
...
159780 0.001316
191560 0.001316
489100 0.001316
407580 0.001316
205600 0.001316
Name: bn_000, Length: 8217, dtype: float64
---------------------------
na 77.248684
1310700 6.035526
0 2.503947
235860 0.011842
221400 0.011842
...
227700 0.001316
150480 0.001316
373620 0.001316
220300 0.001316
253860 0.001316
Name: bo_000, Length: 6829, dtype: float64
---------------------------
na 79.553947
1310700 6.196053
0 2.678947
161080 0.011842
289660 0.011842
...
305800 0.001316
254360 0.001316
129540 0.001316
149040 0.001316
329580 0.001316
Name: bp_000, Length: 5864, dtype: float64
---------------------------
na 81.188158
1310700 6.313158
0 2.876316
220000 0.011842
165940 0.010526
...
289340 0.001316
82580 0.001316
415600 0.001316
410800 0.001316
318260 0.001316
Name: bq_000, Length: 5117, dtype: float64
---------------------------
na 82.096053
1310700 6.411842
0 3.010526
205800 0.010526
229360 0.010526
...
317040 0.001316
442840 0.001316
532420 0.001316
305180 0.001316
343460 0.001316
Name: br_000, Length: 4608, dtype: float64
---------------------------
na 1.221053
0 0.196053
7080 0.060526
10460 0.056579
8360 0.056579
...
176000 0.001316
482500 0.001316
120540 0.001316
113100 0.001316
268920 0.001316
Name: bs_000, Length: 13715, dtype: float64
---------------------------
na 0.256579
0 0.206579
8.4 0.019737
8.65 0.019737
11.67 0.019737
...
2716.9 0.001316
797.75 0.001316
29682.17 0.001316
60.25 0.001316
109.52 0.001316
Name: bt_000, Length: 54524, dtype: float64
---------------------------
na 1.159211
0 0.175000
131080 0.011842
18258 0.010526
32252 0.009211
...
249442 0.001316
828076 0.001316
2867410 0.001316
3207724 0.001316
87120 0.001316
Name: bu_000, Length: 59654, dtype: float64
---------------------------
na 1.159211
0 0.175000
131080 0.011842
18258 0.010526
32252 0.009211
...
249442 0.001316
828076 0.001316
2867410 0.001316
3207724 0.001316
87120 0.001316
Name: bv_000, Length: 59650, dtype: float64
---------------------------
na 5.425000
202 0.059211
212 0.052632
200 0.047368
216 0.046053
...
7306846 0.001316
12010 0.001316
4252774 0.001316
3443966 0.001316
85224 0.001316
Name: bx_000, Length: 66035, dtype: float64
---------------------------
0 2.040789
na 0.763158
26 0.759211
24 0.736842
23 0.694737
...
13844 0.001316
26146 0.001316
7623 0.001316
19424 0.001316
95419 0.001316
Name: by_000, Length: 25857, dtype: float64
---------------------------
0 22.982895
na 4.586842
6 0.378947
12 0.338158
10 0.310526
...
2042542 0.001316
31446 0.001316
140706 0.001316
98230 0.001316
21300 0.001316
Name: bz_000, Length: 19261, dtype: float64
---------------------------
na 7.318421
0 0.177632
1880 0.032895
2212 0.030263
1610 0.028947
...
19466 0.001316
55140 0.001316
38998 0.001316
38536 0.001316
115200 0.001316
Name: ca_000, Length: 31827, dtype: float64
---------------------------
na 1.221053
0 0.177632
19020 0.032895
19920 0.031579
16640 0.028947
...
835960 0.001316
719200 0.001316
646960 0.001316
397260 0.001316
674580 0.001316
Name: cb_000, Length: 33990, dtype: float64
---------------------------
na 5.421053
0 1.468421
6640 0.014474
7018 0.013158
5406 0.011842
...
55439680 0.001316
34782 0.001316
165198 0.001316
4358710 0.001316
32324 0.001316
Name: cc_000, Length: 52486, dtype: float64
---------------------------
1209600 98.867105
na 1.132895
Name: cd_000, dtype: float64
---------------------------
0 19.748684
na 4.197368
246 0.086842
244 0.082895
248 0.082895
...
82246 0.001316
319510 0.001316
75684 0.001316
152228 0.001316
148986 0.001316
Name: ce_000, Length: 25506, dtype: float64
---------------------------
0 35.717105
2 28.875000
na 24.792105
4 4.825000
6 1.430263
...
12862 0.001316
1374 0.001316
5618 0.001316
982 0.001316
1020 0.001316
Name: cf_000, Length: 584, dtype: float64
---------------------------
na 24.792105
4 4.850000
0 4.803947
2 4.700000
6 4.113158
...
554 0.001316
2254 0.001316
1652 0.001316
1352 0.001316
1174 0.001316
Name: cg_000, Length: 717, dtype: float64
---------------------------
0 75.190789
na 24.792105
2 0.017105
Name: ch_000, dtype: float64
---------------------------
na 0.557895
0 0.360526
1692.48 0.015789
1830.72 0.015789
1402.56 0.014474
...
3191645.76 0.001316
2337168.96 0.001316
5041896 0.001316
1705176 0.001316
10367.04 0.001316
Name: ci_000, Length: 55004, dtype: float64
---------------------------
0 78.521053
na 0.557895
3.84 0.172368
2.88 0.159211
6.72 0.147368
...
1655466.24 0.001316
779.52 0.001316
28949.76 0.001316
31127.04 0.001316
12345.6 0.001316
Name: cj_000, Length: 9092, dtype: float64
---------------------------
na 0.557895
0 0.314474
21732.48 0.013158
22427.52 0.011842
2170.56 0.011842
...
63864.96 0.001316
6436.8 0.001316
3717707.52 0.001316
530420.16 0.001316
20952 0.001316
Name: ck_000, Length: 53628, dtype: float64
---------------------------
0 61.368421
na 15.805263
2 4.292105
4 2.305263
6 1.378947
...
2388 0.001316
2108 0.001316
3034 0.001316
4818 0.001316
608 0.001316
Name: cl_000, Length: 1090, dtype: float64
---------------------------
0 40.509211
na 16.388158
14 1.397368
12 1.359211
16 1.355263
...
5952 0.001316
22156 0.001316
11300 0.001316
5940 0.001316
3384 0.001316
Name: cm_000, Length: 2347, dtype: float64
---------------------------
0 95.452632
na 1.159211
2 0.019737
10 0.011842
14 0.010526
...
8016 0.001316
117876 0.001316
1264 0.001316
94960 0.001316
22916 0.001316
Name: cn_000, Length: 1874, dtype: float64
---------------------------
0 83.969737
na 1.159211
2 0.050000
4 0.042105
10 0.034211
...
84572 0.001316
3058 0.001316
101672 0.001316
285452 0.001316
554744 0.001316
Name: cn_001, Length: 6457, dtype: float64
---------------------------
0 50.807895
na 1.159211
6 0.076316
4 0.073684
12 0.072368
...
162246 0.001316
51178 0.001316
15656 0.001316
418700 0.001316
8628 0.001316
Name: cn_002, Length: 17135, dtype: float64
---------------------------
0 4.277632
na 1.159211
4 0.046053
2 0.043421
24 0.035526
...
420014 0.001316
712316 0.001316
2052266 0.001316
1028344 0.001316
94218 0.001316
Name: cn_003, Length: 39884, dtype: float64
---------------------------
0 1.850000
na 1.159211
2016 0.022368
1494 0.022368
1664 0.022368
...
708608 0.001316
1074420 0.001316
1142384 0.001316
2083528 0.001316
21662 0.001316
Name: cn_004, Length: 50097, dtype: float64
---------------------------
0 1.964474
na 1.159211
998 0.048684
826 0.046053
502 0.044737
...
427932 0.001316
924524 0.001316
2724418 0.001316
1380270 0.001316
6816 0.001316
Name: cn_005, Length: 46038, dtype: float64
---------------------------
0 4.477632
na 1.159211
10 0.236842
4 0.225000
14 0.210526
...
932344 0.001316
32996 0.001316
749434 0.001316
16490 0.001316
947254 0.001316
Name: cn_006, Length: 38609, dtype: float64
---------------------------
0 15.313158
na 1.159211
6 0.569737
4 0.552632
8 0.548684
...
13326 0.001316
124054 0.001316
58528 0.001316
10712 0.001316
67132 0.001316
Name: cn_007, Length: 25026, dtype: float64
---------------------------
0 25.442105
na 1.159211
4 0.690789
6 0.639474
8 0.634211
...
14668 0.001316
27270 0.001316
36482 0.001316
30742 0.001316
27040 0.001316
Name: cn_008, Length: 11140, dtype: float64
---------------------------
0 43.271053
na 1.159211
10 0.642105
8 0.619737
12 0.605263
...
19232 0.001316
18108 0.001316
36074 0.001316
17408 0.001316
3328 0.001316
Name: cn_009, Length: 3431, dtype: float64
---------------------------
na 24.792105
0 21.293421
2 6.892105
4 4.203947
6 3.215789
...
1792 0.001316
11350 0.001316
1310 0.001316
4344 0.001316
4316 0.001316
Name: co_000, Length: 2046, dtype: float64
---------------------------
0 15.817105
4 8.242105
6 7.823684
8 5.428947
na 4.588158
...
7158 0.001316
16004 0.001316
31276 0.001316
2872 0.001316
4336 0.001316
Name: cp_000, Length: 2570, dtype: float64
---------------------------
na 1.159211
0 0.175000
131080 0.011842
18258 0.010526
32252 0.009211
...
1135132 0.001316
249442 0.001316
828076 0.001316
2867410 0.001316
87120 0.001316
Name: cq_000, Length: 59652, dtype: float64
---------------------------
na 77.226316
0 22.648684
44 0.006579
54 0.002632
1416 0.002632
...
3074 0.001316
5826 0.001316
574 0.001316
10924 0.001316
9224 0.001316
Name: cr_000, Length: 87, dtype: float64
---------------------------
0 1.822368
na 1.128947
1132 0.135526
1148 0.126316
1186 0.121053
...
41824 0.001316
44884 0.001316
23450 0.001316
11334 0.001316
19828 0.001316
Name: cs_000, Length: 10173, dtype: float64
---------------------------
0 2.092105
12 2.059211
14 2.013158
10 1.944737
16 1.914474
...
10252 0.001316
3962 0.001316
16478 0.001316
6616 0.001316
11026 0.001316
Name: cs_001, Length: 3715, dtype: float64
---------------------------
0 1.953947
12 1.177632
na 1.128947
18 1.107895
16 1.073684
...
60464 0.001316
393982 0.001316
53740 0.001316
48466 0.001316
357990 0.001316
Name: cs_002, Length: 33135, dtype: float64
---------------------------
0 1.998684
na 1.128947
8 0.828947
6 0.760526
10 0.752632
...
123672 0.001316
101534 0.001316
749446 0.001316
420448 0.001316
397290 0.001316
Name: cs_003, Length: 41632, dtype: float64
---------------------------
0 2.038158
na 1.128947
4 0.305263
2 0.247368
6 0.227632
...
147272 0.001316
123484 0.001316
128722 0.001316
17208 0.001316
522352 0.001316
Name: cs_004, Length: 40804, dtype: float64
---------------------------
0 1.886842
na 1.128947
2 0.031579
726 0.023684
1024 0.023684
...
1457070 0.001316
61274 0.001316
1652648 0.001316
412120 0.001316
8598 0.001316
Name: cs_005, Length: 50809, dtype: float64
---------------------------
0 1.882895
na 1.128947
310 0.025000
322 0.025000
356 0.023684
...
6440920 0.001316
2880 0.001316
3398496 0.001316
137992 0.001316
15958 0.001316
Name: cs_006, Length: 48558, dtype: float64
---------------------------
0 8.527632
na 1.128947
2 0.360526
4 0.206579
6 0.157895
...
37276 0.001316
47702 0.001316
95682 0.001316
68404 0.001316
44508 0.001316
Name: cs_007, Length: 18940, dtype: float64
---------------------------
0 23.467105
2 3.528947
4 2.280263
6 1.831579
8 1.515789
...
1746 0.001316
10508 0.001316
1058 0.001316
1128 0.001316
3302 0.001316
Name: cs_008, Length: 821, dtype: float64
---------------------------
0 94.025000
2 2.694737
na 1.128947
4 0.963158
6 0.418421
...
68912 0.001316
95968 0.001316
11270 0.001316
402502 0.001316
84 0.001316
Name: cs_009, Length: 65, dtype: float64
---------------------------
na 23.060526
0 3.613158
2 1.253947
4 1.185526
6 1.085526
...
10288 0.001316
19218 0.001316
3510 0.001316
4364 0.001316
1778 0.001316
Name: ct_000, Length: 2825, dtype: float64
---------------------------
na 23.060526
0 3.553947
2 0.736842
6 0.697368
4 0.682895
...
7524 0.001316
5020 0.001316
5752 0.001316
6600 0.001316
8344 0.001316
Name: cu_000, Length: 3819, dtype: float64
---------------------------
na 23.060526
0 1.717105
2280 0.021053
2038 0.019737
1442 0.018421
...
1954422 0.001316
1572824 0.001316
4123318 0.001316
2287414 0.001316
12680 0.001316
Name: cv_000, Length: 38942, dtype: float64
---------------------------
na 23.060526
0 2.717105
52 0.278947
60 0.277632
56 0.264474
...
107790 0.001316
363120 0.001316
137624 0.001316
214530 0.001316
1124612 0.001316
Name: cx_000, Length: 29453, dtype: float64
---------------------------
0 65.919737
na 23.060526
54 0.273684
16 0.271053
22 0.269737
...
1632 0.001316
61196 0.001316
776 0.001316
626 0.001316
4336 0.001316
Name: cy_000, Length: 837, dtype: float64
---------------------------
na 23.060526
0 15.096053
2 2.740789
4 1.553947
6 1.123684
...
89918 0.001316
7146 0.001316
26016 0.001316
50310 0.001316
64940 0.001316
Name: cz_000, Length: 12164, dtype: float64
---------------------------
0 76.046053
na 23.060526
170 0.013158
52 0.013158
2 0.011842
...
256 0.001316
5428 0.001316
1552 0.001316
3860 0.001316
688 0.001316
Name: da_000, Length: 291, dtype: float64
---------------------------
0 46.092105
na 23.060526
10 2.318421
12 1.975000
14 1.697368
...
528 0.001316
182 0.001316
2518 0.001316
194 0.001316
244 0.001316
Name: db_000, Length: 150, dtype: float64
---------------------------
na 23.060526
0 1.717105
1974 0.023684
1796 0.021053
2144 0.019737
...
3231596 0.001316
2226580 0.001316
122592 0.001316
48122 0.001316
12768 0.001316
Name: dc_000, Length: 39279, dtype: float64
---------------------------
na 4.198684
0 1.427632
56 0.511842
60 0.507895
50 0.506579
...
5388 0.001316
7924 0.001316
8728 0.001316
11760 0.001316
10796 0.001316
Name: dd_000, Length: 7227, dtype: float64
---------------------------
na 4.589474
0 1.423684
50 1.117105
48 1.086842
52 1.085526
...
10826 0.001316
3400 0.001316
2200 0.001316
1564 0.001316
11812 0.001316
Name: de_000, Length: 2088, dtype: float64
---------------------------
0 91.480263
na 6.713158
10 0.081579
30 0.072368
60 0.046053
...
15660 0.001316
250900 0.001316
2180 0.001316
49440 0.001316
2080 0.001316
Name: df_000, Length: 467, dtype: float64
---------------------------
0 90.625000
na 6.713158
2 0.032895
4 0.018421
8 0.015789
...
34114 0.001316
16776 0.001316
100008 0.001316
3772 0.001316
534 0.001316
Name: dg_000, Length: 1621, dtype: float64
---------------------------
0 81.743421
na 6.713158
10 1.161842
30 0.631579
20 0.609211
...
1640 0.001316
7720 0.001316
5150 0.001316
10980 0.001316
58970 0.001316
Name: dh_000, Length: 1196, dtype: float64
---------------------------
0 76.093421
na 6.710526
4 0.307895
2 0.307895
6 0.260526
...
92034 0.001316
43110 0.001316
5154888 0.001316
51558 0.001316
12576 0.001316
Name: di_000, Length: 6671, dtype: float64
---------------------------
0 92.960526
na 6.711842
10 0.039474
30 0.022368
40 0.015789
...
2640 0.001316
45600 0.001316
190 0.001316
8090 0.001316
440 0.001316
Name: dj_000, Length: 81, dtype: float64
---------------------------
0 92.759211
na 6.711842
4 0.010526
6 0.009211
34 0.007895
...
1171130 0.001316
602318 0.001316
310 0.001316
31916 0.001316
9590 0.001316
Name: dk_000, Length: 314, dtype: float64
---------------------------
0 92.910526
na 6.713158
15202350 0.005263
40 0.005263
180 0.005263
...
7920 0.001316
2172210 0.001316
1260 0.001316
2900 0.001316
1006560 0.001316
Name: dl_000, Length: 218, dtype: float64
---------------------------
0 92.855263
na 6.714474
6 0.005263
28 0.003947
70588 0.003947
...
27956 0.001316
2986 0.001316
5710 0.001316
95552 0.001316
347174 0.001316
Name: dm_000, Length: 279, dtype: float64
---------------------------
na 1.159211
176 0.203947
158 0.201316
148 0.196053
162 0.189474
...
383824 0.001316
84604 0.001316
892696 0.001316
14284 0.001316
40386 0.001316
Name: dn_000, Length: 23936, dtype: float64
---------------------------
0 22.019737
na 4.589474
20 0.801316
40 0.361842
60 0.360526
...
443226 0.001316
95016 0.001316
107558 0.001316
38616 0.001316
58588 0.001316
Name: do_000, Length: 23341, dtype: float64
---------------------------
0 21.151316
na 4.592105
2 1.523684
6 1.075000
4 0.906579
...
32572 0.001316
19176 0.001316
13262 0.001316
168652 0.001316
23628 0.001316
Name: dp_000, Length: 12600, dtype: float64
---------------------------
0 74.017105
na 4.592105
80 0.077632
60 0.076316
100 0.067105
...
351452 0.001316
11986 0.001316
5016090 0.001316
10123612 0.001316
15122 0.001316
Name: dq_000, Length: 9588, dtype: float64
---------------------------
0 73.785526
na 4.592105
118 0.080263
96 0.075000
6 0.073684
...
13608 0.001316
429312 0.001316
8706 0.001316
2820 0.001316
1396102 0.001316
Name: dr_000, Length: 7879, dtype: float64
---------------------------
na 4.593421
0 4.364474
46 0.251316
20 0.250000
40 0.246053
...
49712 0.001316
113838 0.001316
395446 0.001316
81744 0.001316
155596 0.001316
Name: ds_000, Length: 30564, dtype: float64
---------------------------
na 4.593421
0 4.186842
14 0.782895
12 0.778947
16 0.756579
...
56870 0.001316
32246 0.001316
69666 0.001316
39102 0.001316
72914 0.001316
Name: dt_000, Length: 17597, dtype: float64
---------------------------
0 5.419737
na 4.592105
80 0.296053
40 0.276316
20 0.264474
...
10690380 0.001316
42321200 0.001316
282480 0.001316
6559160 0.001316
7124860 0.001316
Name: du_000, Length: 33451, dtype: float64
---------------------------
0 5.413158
na 4.592105
8 0.275000
4 0.273684
2 0.264474
...
36526 0.001316
928234 0.001316
433876 0.001316
17268 0.001316
724280 0.001316
Name: dv_000, Length: 35506, dtype: float64
---------------------------
0 65.830263
na 4.588158
2 0.282895
4 0.132895
6 0.082895
...
107140 0.001316
82500 0.001316
69438 0.001316
39104 0.001316
83504 0.001316
Name: dx_000, Length: 18080, dtype: float64
---------------------------
0 60.359211
na 4.589474
8 1.780263
12 1.175000
4 1.142105
...
94798 0.001316
14154 0.001316
31720 0.001316
6274 0.001316
48006 0.001316
Name: dy_000, Length: 7343, dtype: float64
---------------------------
0 94.923684
na 4.585526
2 0.184211
4 0.059211
6 0.040789
8 0.028947
14 0.027632
12 0.019737
10 0.015789
42 0.007895
16 0.007895
52 0.005263
34 0.005263
30 0.005263
38 0.005263
78 0.003947
22 0.003947
104 0.003947
48 0.003947
32 0.003947
194 0.002632
428 0.002632
20 0.002632
18 0.002632
904 0.002632
28 0.002632
274 0.002632
44 0.002632
64 0.002632
162 0.002632
182 0.002632
62 0.002632
178 0.001316
348 0.001316
86 0.001316
96 0.001316
118 0.001316
940 0.001316
276 0.001316
136 0.001316
112 0.001316
1212 0.001316
36 0.001316
122 0.001316
92 0.001316
514 0.001316
106 0.001316
1414 0.001316
324 0.001316
154 0.001316
512 0.001316
192 0.001316
Name: dz_000, dtype: float64
---------------------------
0 94.142105
na 4.585526
2 0.257895
4 0.146053
6 0.081579
...
2134 0.001316
702 0.001316
1606 0.001316
252 0.001316
268 0.001316
Name: ea_000, Length: 142, dtype: float64
---------------------------
0 35.619737
na 6.711842
228290 0.011842
10611120 0.010526
752140 0.010526
...
325780 0.001316
2301060 0.001316
726990 0.001316
411610 0.001316
10074490 0.001316
Name: eb_000, Length: 33326, dtype: float64
---------------------------
na 16.951316
0 2.160526
0.72 0.142105
0.7 0.136842
0.82 0.131579
...
215.6 0.001316
2046.6 0.001316
2034.08 0.001316
541.32 0.001316
1546.74 0.001316
Name: ec_00, Length: 36264, dtype: float64
---------------------------
na 15.805263
0 2.413158
46 0.781579
42 0.728947
48 0.685526
...
9906 0.001316
6930 0.001316
6460 0.001316
12894 0.001316
5120 0.001316
Name: ed_000, Length: 4315, dtype: float64
---------------------------
0 1.819737
na 1.135526
3924 0.018421
3590 0.017105
4468 0.017105
...
552252 0.001316
164860 0.001316
711826 0.001316
205024 0.001316
883900 0.001316
Name: ee_000, Length: 49557, dtype: float64
---------------------------
0 1.923684
na 1.135526
1494 0.027632
1120 0.026316
1198 0.025000
...
262784 0.001316
590804 0.001316
668858 0.001316
31944 0.001316
983308 0.001316
Name: ee_001, Length: 45033, dtype: float64
---------------------------
0 2.263158
na 1.135526
32 0.247368
34 0.243421
30 0.240789
...
620910 0.001316
615478 0.001316
484666 0.001316
584106 0.001316
632658 0.001316
Name: ee_002, Length: 40697, dtype: float64
---------------------------
0 2.582895
na 1.135526
10 0.782895
8 0.748684
12 0.682895
...
115118 0.001316
667486 0.001316
14630 0.001316
470176 0.001316
201836 0.001316
Name: ee_003, Length: 37292, dtype: float64
---------------------------
0 2.981579
na 1.135526
20 0.410526
24 0.388158
16 0.384211
...
200252 0.001316
396032 0.001316
717090 0.001316
444810 0.001316
510354 0.001316
Name: ee_004, Length: 41658, dtype: float64
---------------------------
0 4.669737
na 1.135526
28 0.523684
22 0.511842
20 0.500000
...
126782 0.001316
241850 0.001316
51204 0.001316
18846 0.001316
373918 0.001316
Name: ee_005, Length: 43052, dtype: float64
---------------------------
0 7.380263
na 1.135526
4 0.527632
2 0.518421
8 0.436842
...
142192 0.001316
175186 0.001316
266188 0.001316
169088 0.001316
349840 0.001316
Name: ee_006, Length: 37561, dtype: float64
---------------------------
0 13.103947
na 1.135526
2 0.677632
4 0.531579
6 0.517105
...
304242 0.001316
114244 0.001316
143014 0.001316
247310 0.001316
317840 0.001316
Name: ee_007, Length: 35860, dtype: float64
---------------------------
0 28.657895
na 1.135526
2 0.489474
4 0.388158
10 0.351316
...
80118 0.001316
80158 0.001316
133528 0.001316
55044 0.001316
960024 0.001316
Name: ee_008, Length: 28413, dtype: float64
---------------------------
0 53.019737
na 1.135526
2 0.501316
10 0.500000
8 0.455263
...
18110 0.001316
3380 0.001316
96370 0.001316
238512 0.001316
25566 0.001316
Name: ee_009, Length: 10948, dtype: float64
---------------------------
0 94.984211
na 4.586842
2 0.221053
4 0.082895
6 0.028947
8 0.021053
12 0.011842
10 0.009211
84 0.007895
166 0.005263
20 0.005263
14 0.003947
350 0.002632
22 0.002632
144 0.002632
18 0.002632
26 0.002632
482 0.001316
134 0.001316
276 0.001316
252 0.001316
320 0.001316
82 0.001316
30 0.001316
236 0.001316
40 0.001316
86 0.001316
362 0.001316
74 0.001316
340 0.001316
28 0.001316
Name: ef_000, dtype: float64
---------------------------
0 94.606579
na 4.585526
2 0.298684
4 0.132895
6 0.092105
8 0.052632
10 0.035526
12 0.022368
14 0.021053
26 0.018421
20 0.010526
32 0.007895
48 0.006579
16 0.006579
22 0.005263
28 0.005263
94 0.005263
222 0.005263
68 0.005263
86 0.003947
152 0.003947
34 0.003947
70 0.003947
56 0.003947
36 0.003947
62 0.002632
72 0.002632
42 0.002632
40 0.002632
46 0.002632
430 0.002632
182 0.002632
24 0.002632
164 0.002632
92 0.001316
606 0.001316
88 0.001316
416 0.001316
66 0.001316
124 0.001316
1720 0.001316
44 0.001316
108 0.001316
910 0.001316
1146 0.001316
200 0.001316
162 0.001316
18 0.001316
102 0.001316
80 0.001316
172 0.001316
78 0.001316
892 0.001316
144 0.001316
54 0.001316
168 0.001316
Name: eg_000, dtype: float64
---------------------------
df.replace(to_replace=['na','nan'], value=np.NaN, inplace=True)
df.isna().sum()
class 0
aa_000 0
ab_000 58692
ac_000 4261
ad_000 18842
...
ee_007 863
ee_008 863
ee_009 863
ef_000 3486
eg_000 3485
Length: 171, dtype: int64
nul_values = pd.DataFrame(df.isnull().sum(), columns = ["No.of null values"])
nul_values.loc[:, '%of null values'] = np.round(nul_values.loc[:,'No.of null values']/ df.shape[0]*100, 2)
nul_values.loc[nul_values.loc[:,"No.of null values"]>0, :].sort_values(by = "No.of null values", ascending = True)
pd.set_option('display.max_rows', None)
nul_values
| No.of null values | %of null values | |
|---|---|---|
| class | 0 | 0.00 |
| aa_000 | 0 | 0.00 |
| ab_000 | 58692 | 77.23 |
| ac_000 | 4261 | 5.61 |
| ad_000 | 18842 | 24.79 |
| ae_000 | 3190 | 4.20 |
| af_000 | 3190 | 4.20 |
| ag_000 | 860 | 1.13 |
| ag_001 | 860 | 1.13 |
| ag_002 | 860 | 1.13 |
| ag_003 | 860 | 1.13 |
| ag_004 | 860 | 1.13 |
| ag_005 | 860 | 1.13 |
| ag_006 | 860 | 1.13 |
| ag_007 | 860 | 1.13 |
| ag_008 | 860 | 1.13 |
| ag_009 | 860 | 1.13 |
| ah_000 | 820 | 1.08 |
| ai_000 | 792 | 1.04 |
| aj_000 | 792 | 1.04 |
| ak_000 | 5598 | 7.37 |
| al_000 | 811 | 1.07 |
| am_0 | 792 | 1.04 |
| an_000 | 811 | 1.07 |
| ao_000 | 751 | 0.99 |
| ap_000 | 811 | 1.07 |
| aq_000 | 751 | 0.99 |
| ar_000 | 3487 | 4.59 |
| as_000 | 792 | 1.04 |
| at_000 | 792 | 1.04 |
| au_000 | 792 | 1.04 |
| av_000 | 3188 | 4.19 |
| ax_000 | 3189 | 4.20 |
| ay_000 | 863 | 1.14 |
| ay_001 | 863 | 1.14 |
| ay_002 | 863 | 1.14 |
| ay_003 | 863 | 1.14 |
| ay_004 | 863 | 1.14 |
| ay_005 | 863 | 1.14 |
| ay_006 | 863 | 1.14 |
| ay_007 | 863 | 1.14 |
| ay_008 | 863 | 1.14 |
| ay_009 | 863 | 1.14 |
| az_000 | 863 | 1.14 |
| az_001 | 863 | 1.14 |
| az_002 | 863 | 1.14 |
| az_003 | 863 | 1.14 |
| az_004 | 863 | 1.14 |
| az_005 | 863 | 1.14 |
| az_006 | 863 | 1.14 |
| az_007 | 863 | 1.14 |
| az_008 | 863 | 1.14 |
| az_009 | 863 | 1.14 |
| ba_000 | 881 | 1.16 |
| ba_001 | 881 | 1.16 |
| ba_002 | 881 | 1.16 |
| ba_003 | 881 | 1.16 |
| ba_004 | 881 | 1.16 |
| ba_005 | 881 | 1.16 |
| ba_006 | 881 | 1.16 |
| ba_007 | 881 | 1.16 |
| ba_008 | 881 | 1.16 |
| ba_009 | 881 | 1.16 |
| bb_000 | 820 | 1.08 |
| bc_000 | 3489 | 4.59 |
| bd_000 | 3491 | 4.59 |
| be_000 | 3193 | 4.20 |
| bf_000 | 3189 | 4.20 |
| bg_000 | 811 | 1.07 |
| bh_000 | 811 | 1.07 |
| bi_000 | 751 | 0.99 |
| bj_000 | 751 | 0.99 |
| bk_000 | 29128 | 38.33 |
| bl_000 | 34503 | 45.40 |
| bm_000 | 50095 | 65.91 |
| bn_000 | 55722 | 73.32 |
| bo_000 | 58709 | 77.25 |
| bp_000 | 60461 | 79.55 |
| bq_000 | 61703 | 81.19 |
| br_000 | 62393 | 82.10 |
| bs_000 | 928 | 1.22 |
| bt_000 | 195 | 0.26 |
| bu_000 | 881 | 1.16 |
| bv_000 | 881 | 1.16 |
| bx_000 | 4123 | 5.42 |
| by_000 | 580 | 0.76 |
| bz_000 | 3486 | 4.59 |
| ca_000 | 5562 | 7.32 |
| cb_000 | 928 | 1.22 |
| cc_000 | 4120 | 5.42 |
| cd_000 | 861 | 1.13 |
| ce_000 | 3190 | 4.20 |
| cf_000 | 18842 | 24.79 |
| cg_000 | 18842 | 24.79 |
| ch_000 | 18842 | 24.79 |
| ci_000 | 424 | 0.56 |
| cj_000 | 424 | 0.56 |
| ck_000 | 424 | 0.56 |
| cl_000 | 12012 | 15.81 |
| cm_000 | 12455 | 16.39 |
| cn_000 | 881 | 1.16 |
| cn_001 | 881 | 1.16 |
| cn_002 | 881 | 1.16 |
| cn_003 | 881 | 1.16 |
| cn_004 | 881 | 1.16 |
| cn_005 | 881 | 1.16 |
| cn_006 | 881 | 1.16 |
| cn_007 | 881 | 1.16 |
| cn_008 | 881 | 1.16 |
| cn_009 | 881 | 1.16 |
| co_000 | 18842 | 24.79 |
| cp_000 | 3487 | 4.59 |
| cq_000 | 881 | 1.16 |
| cr_000 | 58692 | 77.23 |
| cs_000 | 858 | 1.13 |
| cs_001 | 858 | 1.13 |
| cs_002 | 858 | 1.13 |
| cs_003 | 858 | 1.13 |
| cs_004 | 858 | 1.13 |
| cs_005 | 858 | 1.13 |
| cs_006 | 858 | 1.13 |
| cs_007 | 858 | 1.13 |
| cs_008 | 858 | 1.13 |
| cs_009 | 858 | 1.13 |
| ct_000 | 17526 | 23.06 |
| cu_000 | 17526 | 23.06 |
| cv_000 | 17526 | 23.06 |
| cx_000 | 17526 | 23.06 |
| cy_000 | 17526 | 23.06 |
| cz_000 | 17526 | 23.06 |
| da_000 | 17526 | 23.06 |
| db_000 | 17526 | 23.06 |
| dc_000 | 17526 | 23.06 |
| dd_000 | 3191 | 4.20 |
| de_000 | 3488 | 4.59 |
| df_000 | 5102 | 6.71 |
| dg_000 | 5102 | 6.71 |
| dh_000 | 5102 | 6.71 |
| di_000 | 5100 | 6.71 |
| dj_000 | 5101 | 6.71 |
| dk_000 | 5101 | 6.71 |
| dl_000 | 5102 | 6.71 |
| dm_000 | 5103 | 6.71 |
| dn_000 | 881 | 1.16 |
| do_000 | 3488 | 4.59 |
| dp_000 | 3490 | 4.59 |
| dq_000 | 3490 | 4.59 |
| dr_000 | 3490 | 4.59 |
| ds_000 | 3491 | 4.59 |
| dt_000 | 3491 | 4.59 |
| du_000 | 3490 | 4.59 |
| dv_000 | 3490 | 4.59 |
| dx_000 | 3487 | 4.59 |
| dy_000 | 3488 | 4.59 |
| dz_000 | 3485 | 4.59 |
| ea_000 | 3485 | 4.59 |
| eb_000 | 5101 | 6.71 |
| ec_00 | 12883 | 16.95 |
| ed_000 | 12012 | 15.81 |
| ee_000 | 863 | 1.14 |
| ee_001 | 863 | 1.14 |
| ee_002 | 863 | 1.14 |
| ee_003 | 863 | 1.14 |
| ee_004 | 863 | 1.14 |
| ee_005 | 863 | 1.14 |
| ee_006 | 863 | 1.14 |
| ee_007 | 863 | 1.14 |
| ee_008 | 863 | 1.14 |
| ee_009 | 863 | 1.14 |
| ef_000 | 3486 | 4.59 |
| eg_000 | 3485 | 4.59 |
df['class'] = df['class'].map({'neg':0, 'pos':1})
df['class'].value_counts()
0 74625 1 1375 Name: class, dtype: int64
sns.countplot(df['class'])
<AxesSubplot:xlabel='class', ylabel='count'>
drop_column = df.columns[df.isnull().mean()>0.3]
drop_column
Index(['ab_000', 'bk_000', 'bl_000', 'bm_000', 'bn_000', 'bo_000', 'bp_000',
'bq_000', 'br_000', 'cr_000'],
dtype='object')
df.drop(drop_column, axis=1, inplace=True)
df.shape
(76000, 161)
num_fs = [feature for feature in df.columns if df[feature].dtype != 'O']
cat_fs = [feature for feature in df.columns if df[feature].dtype == 'O']
print("we have {} numerical features :{}".format(len(num_fs), num_fs))
print("we have {} categorical features :{}".format(len(cat_fs), cat_fs))
we have 2 numerical features :['class', 'aa_000'] we have 159 categorical features :['ac_000', 'ad_000', 'ae_000', 'af_000', 'ag_000', 'ag_001', 'ag_002', 'ag_003', 'ag_004', 'ag_005', 'ag_006', 'ag_007', 'ag_008', 'ag_009', 'ah_000', 'ai_000', 'aj_000', 'ak_000', 'al_000', 'am_0', 'an_000', 'ao_000', 'ap_000', 'aq_000', 'ar_000', 'as_000', 'at_000', 'au_000', 'av_000', 'ax_000', 'ay_000', 'ay_001', 'ay_002', 'ay_003', 'ay_004', 'ay_005', 'ay_006', 'ay_007', 'ay_008', 'ay_009', 'az_000', 'az_001', 'az_002', 'az_003', 'az_004', 'az_005', 'az_006', 'az_007', 'az_008', 'az_009', 'ba_000', 'ba_001', 'ba_002', 'ba_003', 'ba_004', 'ba_005', 'ba_006', 'ba_007', 'ba_008', 'ba_009', 'bb_000', 'bc_000', 'bd_000', 'be_000', 'bf_000', 'bg_000', 'bh_000', 'bi_000', 'bj_000', 'bs_000', 'bt_000', 'bu_000', 'bv_000', 'bx_000', 'by_000', 'bz_000', 'ca_000', 'cb_000', 'cc_000', 'cd_000', 'ce_000', 'cf_000', 'cg_000', 'ch_000', 'ci_000', 'cj_000', 'ck_000', 'cl_000', 'cm_000', 'cn_000', 'cn_001', 'cn_002', 'cn_003', 'cn_004', 'cn_005', 'cn_006', 'cn_007', 'cn_008', 'cn_009', 'co_000', 'cp_000', 'cq_000', 'cs_000', 'cs_001', 'cs_002', 'cs_003', 'cs_004', 'cs_005', 'cs_006', 'cs_007', 'cs_008', 'cs_009', 'ct_000', 'cu_000', 'cv_000', 'cx_000', 'cy_000', 'cz_000', 'da_000', 'db_000', 'dc_000', 'dd_000', 'de_000', 'df_000', 'dg_000', 'dh_000', 'di_000', 'dj_000', 'dk_000', 'dl_000', 'dm_000', 'dn_000', 'do_000', 'dp_000', 'dq_000', 'dr_000', 'ds_000', 'dt_000', 'du_000', 'dv_000', 'dx_000', 'dy_000', 'dz_000', 'ea_000', 'eb_000', 'ec_00', 'ed_000', 'ee_000', 'ee_001', 'ee_002', 'ee_003', 'ee_004', 'ee_005', 'ee_006', 'ee_007', 'ee_008', 'ee_009', 'ef_000', 'eg_000']
for col in df:
df[col].fillna(df[col].median(), inplace=True)
df.isnull().sum()
class 0 aa_000 0 ac_000 0 ad_000 0 ae_000 0 af_000 0 ag_000 0 ag_001 0 ag_002 0 ag_003 0 ag_004 0 ag_005 0 ag_006 0 ag_007 0 ag_008 0 ag_009 0 ah_000 0 ai_000 0 aj_000 0 ak_000 0 al_000 0 am_0 0 an_000 0 ao_000 0 ap_000 0 aq_000 0 ar_000 0 as_000 0 at_000 0 au_000 0 av_000 0 ax_000 0 ay_000 0 ay_001 0 ay_002 0 ay_003 0 ay_004 0 ay_005 0 ay_006 0 ay_007 0 ay_008 0 ay_009 0 az_000 0 az_001 0 az_002 0 az_003 0 az_004 0 az_005 0 az_006 0 az_007 0 az_008 0 az_009 0 ba_000 0 ba_001 0 ba_002 0 ba_003 0 ba_004 0 ba_005 0 ba_006 0 ba_007 0 ba_008 0 ba_009 0 bb_000 0 bc_000 0 bd_000 0 be_000 0 bf_000 0 bg_000 0 bh_000 0 bi_000 0 bj_000 0 bs_000 0 bt_000 0 bu_000 0 bv_000 0 bx_000 0 by_000 0 bz_000 0 ca_000 0 cb_000 0 cc_000 0 cd_000 0 ce_000 0 cf_000 0 cg_000 0 ch_000 0 ci_000 0 cj_000 0 ck_000 0 cl_000 0 cm_000 0 cn_000 0 cn_001 0 cn_002 0 cn_003 0 cn_004 0 cn_005 0 cn_006 0 cn_007 0 cn_008 0 cn_009 0 co_000 0 cp_000 0 cq_000 0 cs_000 0 cs_001 0 cs_002 0 cs_003 0 cs_004 0 cs_005 0 cs_006 0 cs_007 0 cs_008 0 cs_009 0 ct_000 0 cu_000 0 cv_000 0 cx_000 0 cy_000 0 cz_000 0 da_000 0 db_000 0 dc_000 0 dd_000 0 de_000 0 df_000 0 dg_000 0 dh_000 0 di_000 0 dj_000 0 dk_000 0 dl_000 0 dm_000 0 dn_000 0 do_000 0 dp_000 0 dq_000 0 dr_000 0 ds_000 0 dt_000 0 du_000 0 dv_000 0 dx_000 0 dy_000 0 dz_000 0 ea_000 0 eb_000 0 ec_00 0 ed_000 0 ee_000 0 ee_001 0 ee_002 0 ee_003 0 ee_004 0 ee_005 0 ee_006 0 ee_007 0 ee_008 0 ee_009 0 ef_000 0 eg_000 0 dtype: int64
for i in cat_fs:
if df[i].dtype == 'O':
df[i] = df[i].astype('float')
df.info()
<class 'pandas.core.frame.DataFrame'> RangeIndex: 76000 entries, 0 to 75999 Columns: 161 entries, class to eg_000 dtypes: float64(159), int64(2) memory usage: 93.4 MB
num_fs = [feature for feature in df.columns if df[feature].dtype != 'O']
cat_fs = [feature for feature in df.columns if df[feature].dtype == 'O']
print("we have {} numerical features :{}".format(len(num_fs), num_fs))
print('======================================================================')
print("we have {} categorical features :{}".format(len(cat_fs), cat_fs))
we have 161 numerical features :['class', 'aa_000', 'ac_000', 'ad_000', 'ae_000', 'af_000', 'ag_000', 'ag_001', 'ag_002', 'ag_003', 'ag_004', 'ag_005', 'ag_006', 'ag_007', 'ag_008', 'ag_009', 'ah_000', 'ai_000', 'aj_000', 'ak_000', 'al_000', 'am_0', 'an_000', 'ao_000', 'ap_000', 'aq_000', 'ar_000', 'as_000', 'at_000', 'au_000', 'av_000', 'ax_000', 'ay_000', 'ay_001', 'ay_002', 'ay_003', 'ay_004', 'ay_005', 'ay_006', 'ay_007', 'ay_008', 'ay_009', 'az_000', 'az_001', 'az_002', 'az_003', 'az_004', 'az_005', 'az_006', 'az_007', 'az_008', 'az_009', 'ba_000', 'ba_001', 'ba_002', 'ba_003', 'ba_004', 'ba_005', 'ba_006', 'ba_007', 'ba_008', 'ba_009', 'bb_000', 'bc_000', 'bd_000', 'be_000', 'bf_000', 'bg_000', 'bh_000', 'bi_000', 'bj_000', 'bs_000', 'bt_000', 'bu_000', 'bv_000', 'bx_000', 'by_000', 'bz_000', 'ca_000', 'cb_000', 'cc_000', 'cd_000', 'ce_000', 'cf_000', 'cg_000', 'ch_000', 'ci_000', 'cj_000', 'ck_000', 'cl_000', 'cm_000', 'cn_000', 'cn_001', 'cn_002', 'cn_003', 'cn_004', 'cn_005', 'cn_006', 'cn_007', 'cn_008', 'cn_009', 'co_000', 'cp_000', 'cq_000', 'cs_000', 'cs_001', 'cs_002', 'cs_003', 'cs_004', 'cs_005', 'cs_006', 'cs_007', 'cs_008', 'cs_009', 'ct_000', 'cu_000', 'cv_000', 'cx_000', 'cy_000', 'cz_000', 'da_000', 'db_000', 'dc_000', 'dd_000', 'de_000', 'df_000', 'dg_000', 'dh_000', 'di_000', 'dj_000', 'dk_000', 'dl_000', 'dm_000', 'dn_000', 'do_000', 'dp_000', 'dq_000', 'dr_000', 'ds_000', 'dt_000', 'du_000', 'dv_000', 'dx_000', 'dy_000', 'dz_000', 'ea_000', 'eb_000', 'ec_00', 'ed_000', 'ee_000', 'ee_001', 'ee_002', 'ee_003', 'ee_004', 'ee_005', 'ee_006', 'ee_007', 'ee_008', 'ee_009', 'ef_000', 'eg_000'] ====================================================================== we have 0 categorical features :[]
for i in num_fs:
if df[i].dtype != 'O':
df[i]=df[i].astype('int')
df.head()
| class | aa_000 | ac_000 | ad_000 | ae_000 | af_000 | ag_000 | ag_001 | ag_002 | ag_003 | ... | ee_002 | ee_003 | ee_004 | ee_005 | ee_006 | ee_007 | ee_008 | ee_009 | ef_000 | eg_000 | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 0 | 76698 | 2130706438 | 280 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 1240520 | 493384 | 721044 | 469792 | 339156 | 157956 | 73224 | 0 | 0 | 0 |
| 1 | 0 | 33058 | 0 | 128 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 421400 | 178064 | 293306 | 245416 | 133654 | 81140 | 97576 | 1500 | 0 | 0 |
| 2 | 0 | 41040 | 228 | 100 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 277378 | 159812 | 423992 | 409564 | 320746 | 158022 | 95128 | 514 | 0 | 0 |
| 3 | 0 | 12 | 70 | 66 | 0 | 10 | 0 | 0 | 0 | 318 | ... | 240 | 46 | 58 | 44 | 10 | 0 | 0 | 0 | 4 | 32 |
| 4 | 0 | 60874 | 1368 | 458 | 0 | 0 | 0 | 0 | 0 | 0 | ... | 622012 | 229790 | 405298 | 347188 | 286954 | 311560 | 433954 | 1218 | 0 | 0 |
5 rows × 161 columns
x_axis = num_fs
for x in x_axis:
fig, axes = plt.subplots(1,2, figsize=(18,4))
sns.distplot(df[x], ax=axes[0], kde=True, hist=False)
sns.boxplot(df[x], ax=axes[1], orient='h', showmeans=True, palette='Set2')
fig.suptitle('Univariate Analysis of '+x, fontsize=16)
fig = plt.figure(figsize=(20, 850))
for i in range(0, len(num_fs)):
ax = plt.subplot(81, 2, i+1)
sns.regplot(data=df, y='class', x=num_fs[i], color='b')
plt.tight_layout()
sns.heatmap(df.corr(), annot=True, cmap='icefire', linewidths=0.3)
fig = plt.gcf()
fig.set_size_inches(30,30)
plt.title("corr between variable", color='black', size=25)
plt.show()
